#
# Copyright 2008 Qumranet, Inc. All rights reserved.
# Use is subject to license terms.
#
"""
A module containing miscellaneous functions and classes that are user plentifuly around vdsm.

.. attribute:: utils.symbolerror
    
    Contains a reverse dictionary pointing from error string to its error code.
"""
import constants
from SimpleXMLRPCServer import SimpleXMLRPCServer
import SocketServer
import threading
import os, traceback, time
import logging
import errno
import subprocess
import socket
import fcntl
import struct

from config import config

NET_CONF_DIR = '/etc/sysconfig/network-scripts/'
NET_CONF_BACK_DIR = NET_CONF_DIR + '.vdsmback/'

def rmFile(fileToRemove):
    """
    Try to remove a file.

    .. note::
        If the operation fails the function exists silently.
    """
    try: 
        os.unlink(fileToRemove)
    except:
        pass

def readTotalFreeCachedMem(readSwap=False):
    """
    Parse ``/proc/meminfo`` and return some of it.

    For a reason unknown to me, ``/proc/meminfo`` is is sometime
    empty when opened. If that happens, the function retries to open it
    3 times.

    :param readSwap: Whether to return swap sizes instead of RAM sizes.
    :type readSwap: bool
    :returns: a tuple with this information ``(Total Memory, Free Memory, Cached Memmory)``
    """
    # FIXME the root cause for these retries should be found and fixed
    tries = 3
    while True:
        tries -= 1
        try:
            lines = []
            lines = file('/proc/meminfo').readlines()
            for line in lines:
                var, val = line.split()[0:2]
                if 'MemTotal:' == var:
                    memTotal = long(val)
                elif 'MemFree:' == var:
                    memFree = long(val)
                elif 'Cached:' == var:
                    memCached = long(val)
                elif 'Buffers:' == var:
                    memBuffers = long(val)
                elif 'SwapTotal:' == var:
                    swapTotal = long(val)
                elif 'SwapFree:' == var:
                    swapFree = long(val)
            if readSwap:
                return swapTotal, swapFree
            else:
                return (memTotal, memFree, memCached + memBuffers)
        except:
            logging.warning(traceback.format_exc())
            logging.warning(lines)
            if tries <= 0:
                raise
            time.sleep(0.1)

#Threaded version of SimpleXMLRPCServer
class SimpleThreadedXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer):
    allow_reuse_address = True

# Create communication channel to vm's wrappers to regain connection 
def createDaemon():
   """
   Detach this process from the controlling terminal and run it in the
   background as a daemon.
   """
   # Default daemon parameters.
   # File mode creation mask of the daemon.
   UMASK = 007

   # Default working directory for the daemon.
   WORKDIR = "/"

   # Default maximum for the number of available file descriptors.
   MAXFD = 1024

   # The standard I/O file descriptors are redirected to /dev/null by default.
   if (hasattr(os, "devnull")):
      REDIRECT_TO = os.devnull
   else:
      REDIRECT_TO = "/dev/null"
   
   try:
      # Fork a child process so the parent can exit.  This returns control to
      # the command-line or shell.  It also guarantees that the child will not
      # be a process group leader, since the child receives a new process ID
      # and inherits the parent's process group ID.  This step is required
      # to insure that the next call to os.setsid is successful.
      pid = os.fork()
   except OSError, e:
      raise Exception, "%s [%d]" % (e.strerror, e.errno)

   if (pid == 0):	# The first child.
      # To become the session leader of this new session and the process group
      # leader of the new process group, we call os.setsid().  The process is
      # also guaranteed not to have a controlling terminal.
      os.setsid()

      # Is ignoring SIGHUP necessary?
      #
      # It's often suggested that the SIGHUP signal should be ignored before
      # the second fork to avoid premature termination of the process.  The
      # reason is that when the first child terminates, all processes, e.g.
      # the second child, in the orphaned group will be sent a SIGHUP.
      #
      # "However, as part of the session management system, there are exactly
      # two cases where SIGHUP is sent on the death of a process:
      #
      #   1) When the process that dies is the session leader of a session that
      #      is attached to a terminal device, SIGHUP is sent to all processes
      #      in the foreground process group of that terminal device.
      #   2) When the death of a process causes a process group to become
      #      orphaned, and one or more processes in the orphaned group are
      #      stopped, then SIGHUP and SIGCONT are sent to all members of the
      #      orphaned group." [2]
      #
      # The first case can be ignored since the child is guaranteed not to have
      # a controlling terminal.  The second case isn't so easy to dismiss.
      # The process group is orphaned when the first child terminates and
      # POSIX.1 requires that every STOPPED process in an orphaned process
      # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
      # second child is not STOPPED though, we can safely forego ignoring the
      # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
      #
      # import signal           # Set handlers for asynchronous events.
      # signal.signal(signal.SIGHUP, signal.SIG_IGN)

      try:
         # Fork a second child and exit immediately to prevent zombies.  This
         # causes the second child process to be orphaned, making the init
         # process responsible for its cleanup.  And, since the first child is
         # a session leader without a controlling terminal, it's possible for
         # it to acquire one by opening a terminal in the future (System V-
         # based systems).  This second fork guarantees that the child is no
         # longer a session leader, preventing the daemon from ever acquiring
         # a controlling terminal.
         pid = os.fork()	# Fork a second child.
      except OSError, e:
         raise Exception, "%s [%d]" % (e.strerror, e.errno)

      if (pid == 0):	# The second child.
         # Since the current working directory may be a mounted filesystem, we
         # avoid the issue of not being able to unmount the filesystem at
         # shutdown time by changing it to the root directory.
         os.chdir(WORKDIR)
         # We probably don't want the file mode creation mask inherited from
         # the parent, so we give the child complete control over permissions.
         os.umask(UMASK)
      else:
         # exit() or _exit()?  See below.
         os._exit(0)	# Exit parent (the first child) of the second child.
   else:
      # exit() or _exit()?
      # _exit is like exit(), but it doesn't call any functions registered
      # with atexit (and on_exit) or any registered signal handlers.  It also
      # closes any open file descriptors.  Using exit() may cause all stdio
      # streams to be flushed twice and any temporary files may be unexpectedly
      # removed.  It's therefore recommended that child branches of a fork()
      # and the parent branch(es) of a daemon use _exit().
      os._exit(0)	# Exit parent of the first child.

   # Close all open file descriptors.  This prevents the child from keeping
   # open any file descriptors inherited from the parent.  There is a variety
   # of methods to accomplish this task.  Three are listed below.
   #
   # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
   # number of open file descriptors to close.  If it doesn't exists, use
   # the default value (configurable).
   #
   # try:
   #    maxfd = os.sysconf("SC_OPEN_MAX")
   # except (AttributeError, ValueError):
   #    maxfd = MAXFD
   #
   # OR
   #
   # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
   #    maxfd = os.sysconf("SC_OPEN_MAX")
   # else:
   #    maxfd = MAXFD
   #
   # OR
   #
   # Use the getrlimit method to retrieve the maximum file descriptor number
   # that can be opened by this process.  If there is not limit on the
   # resource, use the default value.
   #
   
   import resource  # Resource usage information.
   maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
   if (maxfd == resource.RLIM_INFINITY):
      maxfd = MAXFD
  
   # Iterate through and close all file descriptors.
   for fd in range(0, maxfd):
      try:
         os.close(fd)
      except OSError:	# ERROR, fd wasn't open to begin with (ignored)
         pass

   # Redirect the standard I/O file descriptors to the specified file.  Since
   # the daemon has no controlling terminal, most daemons redirect stdin,
   # stdout, and stderr to /dev/null.  This is done to prevent side-effects
   # from reads and writes to the standard I/O file descriptors.

   # This call to open is guaranteed to return the lowest file descriptor,
   # which will be 0 (stdin), since it was closed above.
   
   os.open(REDIRECT_TO, os.O_RDWR)	# standard input (0)

   # Duplicate standard input to standard output and standard error.
   os.dup2(0, 1)           # standard output (1)
   os.dup2(0, 2)           # standard error (2)

   return(0)

class InterfaceSample:
    """
    A network interface sample.

    The sample is set at the time of initialization and can't be updated.
    """
    def readIfaceStat(self, ifid, stat):
        """
        Get and interface's stat.

        .. note::
            Really ugly implementation; from time to time, Linux returns an empty line.

        :param ifid: The ID of the interface you want to query.
        :param stat: The type of statistic you want get.

        :returns: The value of the statistic you asked for.
        :type: int
        """
        f = '/sys/class/net/%s/statistics/%s' % (ifid, stat)
        tries = 5
        while tries:
            tries -= 1
            try:
                s = file(f).read()
            except IOError, e:
                if e.errno != errno.ENOENT: # silently ignore missing wifi stats
                    logging.debug(traceback.format_exc())
                return 0
            try:
                return int(s)
            except:
                logging.warning(traceback.format_exc())
                logging.warning('bad %s: %s' % (f, s))
                if not tries:
                    raise

    def readIfaceOperstate(self, ifid):
        """
        Return the operational state of the interface.

        :returns: ``'up'`` if interface is up. ``'down'`` or ``0`` id it's down.
        """
        f = '/sys/class/net/%s/operstate' % (ifid,)
        try:
            s = file(f).readline().strip()

            if s != 'unknown':
                return s

            SIOCGIFFLAGS = 0x8913
            helperSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            res = fcntl.ioctl(helperSocket.fileno(), SIOCGIFFLAGS,
                              ifid + '\0'*256)
            flags, = struct.unpack('H', res[16:18])
            if flags & 1:
                return 'up'
            else:
                return 'down'
        except:
            logging.debug(traceback.format_exc())
            return '0'

    def __init__(self, ifid):
        self.rx = self.readIfaceStat(ifid, 'rx_bytes')
        self.tx = self.readIfaceStat(ifid, 'tx_bytes')
        self.rxDropped = self.readIfaceStat(ifid, 'rx_dropped')
        self.txDropped = self.readIfaceStat(ifid, 'tx_dropped')
        self.rxErrors = self.readIfaceStat(ifid, 'rx_errors')
        self.txErrors = self.readIfaceStat(ifid, 'tx_errors')
        self.operstate = self.readIfaceOperstate(ifid)


class TotalCpuSample:
    """
    A sample of total CPU consumption.

    The sample is taken at initialization time and can't be updated.
    """
    def __init__(self):
        self.user, userNice, self.sys, self.idle = \
                        map(int, file('/proc/stat').readline().split()[1:5])
        self.user += userNice

class PidCpuSample:
    """
    A sample of the CPU consumption of a process.

    The sample is taken at initialization time and can't be updated.
    """
    def __init__(self, pid):
        self.user, self.sys = \
                map(int, file('/proc/%s/stat' % pid).read().split()[13:15])

class BaseSample:
    """
    A sample of the statistics for a process.
    """
    def __init__(self, pid, ifids):
        self.timestamp = time.time()
        self.interfaces= {}
        for ifid in ifids:
            self.interfaces[ifid] = InterfaceSample(ifid)
        self.pidcpu = PidCpuSample(pid)

class HostSample(BaseSample):
    """
    A sample of host-related statistics.

    Contatins the sate of the host in the time of initailization.
    """
    MONITORED_PATHS = ['/tmp', '/var/log', '/var/log/core', '/var/vdsm']

    def _getDiskStats(self):
        d = {}
        for p in self.MONITORED_PATHS:
            free = 0
            try:
                stat = os.statvfs(p)
                free = stat.f_bavail * stat.f_bsize / 2**20
            except:
                pass
            d[p] = {'free': str(free)}
        return d

    def __init__(self, pid, ifids):
        """
        Initialize a HostSample.

        :param pid: The PID of this vdsm host.
        :type pid: int
        :param ifids: The IDs of the interfaces you want to sample.
        :type: list
        """
        BaseSample.__init__(self, pid, ifids)
        self.totcpu = TotalCpuSample()
        total, free, cached = readTotalFreeCachedMem()
        self.memUsed = 100 - int(100.0 * (free + cached) / total)
        try:
            self.cpuLoad = file('/proc/loadavg').read().split()[1]
        except:
            self.cpuLoad = '0.0'
        self.diskStats = self._getDiskStats()

class StatsThread(threading.Thread):
    """
    A thread that periodically checks the stats of interfaces
    """
    AVERAGING_WINDOW = 5
    SAMPLE_INTERVAL_SEC = 2
    MBITTOBYTES = 1000000 / 8
    def __init__(self, log, ifids, ifrates, ifmacs, elapsedTimeOffset=0):
        threading.Thread.__init__(self)
        self._log = log
        self._lastCheckTime = 0
        self._stopEvent = threading.Event()
        self.startTime = time.time() - elapsedTimeOffset
        self._samples = []
        self._ifids = ifids
        self._ifrates = ifrates
        self._ifmacs = ifmacs
        self._ncpus = 1
        self._lineRate = (sum(ifrates) or 1000) * 10**6 / 8 # in bytes-per-second

    def stop(self):
        self._stopEvent.set()

    def sample(self): # override
        pass

    def run(self):
        import QemuMonitor
        try:
            while not self._stopEvent.isSet():
                time.sleep(self.SAMPLE_INTERVAL_SEC)
                try:
                    self._samples.append(self.sample())
                    if len(self._samples) > self.AVERAGING_WINDOW:
                        self._samples.pop(0)
                except QemuMonitor.TimeoutError:
                    self._log.error(traceback.format_exc())
        except:
            self._log.error(traceback.format_exc())

    def get(self):
        """
        Compile and return a dict containg the stats.

        :returns: a dict that with the following keys:
            
            * cpuUser
            * cpuSys
            * cpuIdle
            * rxRate
            * txRate
        """
        stats = {'cpuUser': 0.0, 'cpuSys': 0.0, 'cpuIdle': 100.0, 
                 'rxRate': 0.0, 'txRate': 0.0 }
        now = time.time()
        stats['elapsedTime'] = int(now - self.startTime)
        if len(self._samples) < 2:
            return stats
        hs0, hs1 = self._samples[0], self._samples[-1]
        interval = hs1.timestamp - hs0.timestamp

        rx = tx = rxDropped = txDropped = 0
        stats['network'] = {}
        for ifid, ifrate, ifmac in zip(self._ifids, self._ifrates,
                                       self._ifmacs):
            ifrate = ifrate or 1000
            Mbps2Bps = 10**6 / 8
            thisRx = (hs1.interfaces[ifid].rx - hs0.interfaces[ifid].rx) % 2**32
            thisTx = (hs1.interfaces[ifid].tx - hs0.interfaces[ifid].tx) % 2**32
            rxRate = 100.0 * thisRx / interval / ifrate / Mbps2Bps
            txRate = 100.0 * thisTx / interval / ifrate / Mbps2Bps
            if txRate > 100 or rxRate > 100:
                txRate = min(txRate, 100.0)
                rxRate = min(rxRate, 100.0)
                self._log.debug('Rate above 100%%. DEBUG: ifid %s interval: %s thisRx %s thisTx %s samples %s', ifid, interval, thisRx, thisTx, [(hs.timestamp, hs.interfaces[ifid].rx, hs.interfaces[ifid].tx) for hs in self._samples])
            stats['network'][ifid] = {'name': ifid, 'speed': str(ifrate),
                    'rxDropped': str(hs1.interfaces[ifid].rxDropped),
                    'txDropped': str(hs1.interfaces[ifid].txDropped),
                    'rxErrors': str(hs1.interfaces[ifid].rxErrors),
                    'txErrors': str(hs1.interfaces[ifid].txErrors),
                    'state': hs1.interfaces[ifid].operstate,
                    'rxRate': str(rxRate),
                    'txRate': str(txRate),
                    'macAddr': ifmac,
#                    'interval': str(interval), 'thisRx': str(thisRx), 'thisTx': str(thisTx),
#                    'samples': str([(hs.timestamp, hs.interfaces[ifid].rx, hs.interfaces[ifid].tx) for hs in self._samples])
                    }
            rx += thisRx
            tx += thisTx
            rxDropped += hs1.interfaces[ifid].rxDropped
            txDropped += hs1.interfaces[ifid].txDropped
        stats['rxRate'] = 100.0 * rx / interval / self._lineRate
        stats['txRate'] = 100.0 * tx / interval / self._lineRate
        if stats['txRate'] > 100 or stats['rxRate'] > 100:
            stats['txRate'] = min(stats['txRate'], 100.0)
            stats['rxRate'] = min(stats['rxRate'], 100.0)
            logging.debug(stats)
#        stats['rxBps'] = float(rx) / interval
#        stats['txBps'] = float(tx) / interval
        stats['rxDropped'] = rxDropped / interval
        stats['txDropped'] = txDropped / interval

        return stats

class HostStatsThread(StatsThread):
    """
    A thread that periodically samples host statistics.
    """
    def __init__(self, cif, log, ifids, ifrates, elapsedTimeOffset=0):
        StatsThread.__init__(self, log, ifids, ifrates,
                             [''] * len(ifids), # fake ifmacs
                             elapsedTimeOffset)
        self._imagesStatus = ImagePathStatus(cif)
        self._pid = os.getpid()
        self._ncpus = len(os.listdir('/sys/class/cpuid/'))

    def stop(self):
        self._imagesStatus.stop()
        StatsThread.stop(self)

    def _updateIfRates(self, hs0, hs1):
        # import netinfo only after it imported utils
        from netinfo import speed as nicspeed

        i = 0
        for ifid in self._ifids:
            if hs0.interfaces[ifid].operstate != 'up' and \
               hs1.interfaces[ifid].operstate == 'up':
                self._ifrates[i] = nicspeed(ifid)
            i += 1

    def sample(self):
        hs = HostSample(self._pid, self._ifids)
        if self._samples:
            self._updateIfRates(self._samples[-1], hs)
        return hs

    def get(self):
        stats = StatsThread.get(self)
        stats['cpuSysVdsmd'] = stats['cpuUserVdsmd'] = 0.0
        stats['storageDomains'] = {}
        if self._imagesStatus._cif.irs:
            self._imagesStatus._refreshStorageDomains()
        now = time.time()
        for sd, d in self._imagesStatus.storageDomains.iteritems():
            stats['storageDomains'][sd] = {'code': d['code'],
                        'delay': d['delay'],
                        'lastCheck': '%.1f' % (now - d['lastCheck']),
                        'valid': d['valid']}
        if len(self._samples) < 2:
            return stats
        hs0, hs1 = self._samples[0], self._samples[-1]
        interval = hs1.timestamp - hs0.timestamp
        jiffies = (hs1.pidcpu.user - hs0.pidcpu.user) % 2**32
        stats['cpuUserVdsmd'] = (jiffies / interval) % 2**32
        jiffies = hs1.pidcpu.sys - hs0.pidcpu.sys
        stats['cpuSysVdsmd'] = (jiffies / interval) % 2**32

        jiffies = (hs1.totcpu.user - hs0.totcpu.user) % 2**32
        stats['cpuUser'] = jiffies / interval / self._ncpus
        jiffies = (hs1.totcpu.sys - hs0.totcpu.sys) % 2**32
        stats['cpuSys'] = jiffies / interval / self._ncpus
        stats['cpuIdle'] = max(0.0,
                 100.0 - stats['cpuUser'] - stats['cpuSys'])
        stats['memUsed'] = hs1.memUsed
        stats['cpuLoad'] = hs1.cpuLoad

        stats['diskStats'] = hs1.diskStats
        return stats

def convertToStr (val):
    varType = type(val)
    if varType is float:
        return '%.2f' % (val)
    elif varType is int:
        return '%d' % (val)
    else:
        return val

def execAndGetOutput(command):
    p = subprocess.Popen(command, shell=True, close_fds=True,
                stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)
    out, err = p.communicate()
    return out.splitlines(True)

def checkPathStat(pathToCheck):
    try:
        startTime = time.time()
        os.statvfs(pathToCheck)
        delay = time.time() - startTime 
        return (True, delay)
    except:
        return (False, 0)

class ImagePathStatus(threading.Thread):
    def __init__ (self, cif, interval=None):
        if interval is None:
            interval = config.getint('irs', 'images_check_times')
        self._interval = interval
        self._cif = cif
        self.storageDomains = {}
        self._stopEvent = threading.Event()
        threading.Thread.__init__(self, name='ImagePathStatus')
        if self._interval > 0:
            self.start()

    def stop (self):
        self._stopEvent.set()

    def _refreshStorageDomains(self):
        self.storageDomains = self._cif.irs.repoStats()
        del self.storageDomains["status"]
        if "args" in self.storageDomains:
            del self.storageDomains["args"]

    def run (self):
        try:
            while not self._stopEvent.isSet():
                if self._cif.irs:
                    self._refreshStorageDomains()
                self._stopEvent.wait(self._interval)
        except:
            logging.error(traceback.format_exc())

def getPidNiceness(pid):
    """
    Get the nice level of a process.

    :param pid: the PID of the process.
    :type pid: int
    """
    stat = file('/proc/%s/stat' % (pid)).readlines()[0]
    return int(stat.split(') ')[-1].split()[16])

def tobool(s):
    try:
        if s == None:
            return False
        if type(s) == bool:
            return s
        if s.lower() == 'true':
            return True
        return bool(int(s))
    except:
        return False


__hostUUID = ''
def getHostUUID():
    global __hostUUID
    if __hostUUID:
        return __hostUUID

    __hostUUID = 'None'

    try:
        p = subprocess.Popen(constants.EXT_SUDO + " " + constants.EXT_DMIDECODE + " |" +
                             constants.EXT_AWK + " '/^\tUUID: /{ print $2 }'",
                    close_fds=True, shell=True,
                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
        out, err = p.communicate()
        if p.returncode == 0 and 'Not' not in out:
            #Avoid error string - 'Not Settable' or 'Not Present'
            __hostUUID = out.strip()
        else:
            logging.warning('Could not find host UUID.')

        p = subprocess.Popen(constants.EXT_CAT + " /sys/class/net/eth*/address " + "|" + 
                             constants.EXT_SORT + " | " + constants.EXT_HEAD + " --lines=1",
                    close_fds=True, shell=True,
                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
        out, err = p.communicate()
        mac = ""
        if p.returncode == 0:
            mac = out.strip()
        else:
            logging.warning('Could not find host MAC.')
 
        if __hostUUID != "None":
            __hostUUID += "_" + mac
        else:
            __hostUUID = "_" + mac
    except:
        logging.error(traceback.format_exc())

    return __hostUUID

symbolerror = {}
for code, symbol in errno.errorcode.iteritems():
    symbolerror[os.strerror(code)] = symbol
